#!/usr/bin/env python
# coding=utf-8
# __author__ = 'Yunchao Ling'

import os
import sys
import datetime
import time
import pycurl
import StringIO

NOW_DATE = datetime.datetime.now().strftime("%Y%m%d")


def checkSpace(dev):
    import commands
    import re

    space = 10
    line = commands.getoutput("df -B G -P")
    splitline = line.split("\n")
    for item in splitline:
        if item.startswith(dev):
            item = item.rstrip()
            regex = "^.*? *\d*G *\d*G *(\d*)G *\d*% *.*$"
            match = re.match(regex, item)
            if match:
                result = match.group(1)
                space = int(result)
                break
    return space


def Verify_Meta_XML(filename):
    import commands

    cmd = "grep '</PubmedArticleSet>' " + filename
    result = commands.getoutput(cmd)
    if result.find("</PubmedArticleSet>") != -1:
        print filename + " is valid."
        return True
    elif result.find("No such file or directory") != -1:
        return False
    else:
        print filename + " is invalid. Try to redownload..."
        return False


def Get_PMC_Meta(filename):
    ids = ""
    infile = open(filename, "r")
    for line in infile:
        line = line.rstrip()
        ids += line + ","
    infile.close()
    ids = ids[:-1]
    taskid = filename.split(".")[-2]
    xml_name = "pubmedmeta_" + NOW_DATE + "." + taskid + ".xml"
    try:
        url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi'
        buf = StringIO.StringIO()
        c = pycurl.Curl()
        c.setopt(pycurl.POST, 1)
        c.setopt(c.URL, url)
        c.setopt(pycurl.HTTPPOST, [('db', 'pubmed'), ('id', ids), ('retmode', 'xml')])
        c.setopt(c.WRITEFUNCTION, buf.write)
        c.perform()
        outfile = open(xml_name, "w")
        outfile.write(buf.getvalue())
        outfile.close()
        buf.close()
        c.close()
        print xml_name + " has been successfully downloaded."
        while not Verify_Meta_XML(xml_name):
            print("Invalid xml. Try to redownload...")
            if os.path.exists(xml_name):
                os.remove(xml_name)
            Get_PMC_Meta(filename)
        time.sleep(2)
        print("Compressing " + xml_name + " ..."),
        os.system("tar -czf " + xml_name + ".tar.gz " + xml_name + " --remove-files")
        print("complete.")
        time.sleep(2)
        print("Calculating md5 ..."),
        os.system("md5sum " + xml_name + ".tar.gz >" + xml_name + ".tar.gz.md5")
        print("complete.")
    except pycurl.error:
        print("Network error. Try to redownload...")
        if os.path.exists(xml_name):
            os.remove(xml_name)
        Get_PMC_Meta(filename)


if __name__ == '__main__':
    print("Loading date, current given date is ..."),
    if len(sys.argv) >= 2:
        NOW_DATE = sys.argv[1]
    print(NOW_DATE)
    print("Loading task files ..."),
    filelist = os.listdir(".")
    print("complete.")
    for filename in filelist:
        if filename.endswith(".task"):
            while checkSpace("/dev/xvda") < 5:
                time.sleep(30)
            print("Start downloading task " + filename + " ...")
            Get_PMC_Meta(filename)
            print("Deleting task file ..."),
            if os.path.exists(filename):
                os.remove(filename)
            print("complete.")
            print("Task " + filename + " complete.")
    print("Deleting todo list file ..."),
    if os.path.exists("pubmedid_todo_" + NOW_DATE + ".txt"):
        os.remove("pubmedid_todo_" + NOW_DATE + ".txt")
    print("complete.")
    print("Setting complete symbol ..."),
    os.system("touch all.complete")
    print("complete.")
    print("All tasks completed.")
